import os
import time
#from datetime import date, tzinfo, timedelta, datetime
import datetime

import pandas as pd
import matplotlib.pyplot as plt
from pandas import *
from utility import *

__doc__="""
Prepare file for backtesting from the raw tick data file
"""
strtoday  = datetime.datetime.now().strftime("%Y-%m-%d")
outputdir_= 'C:\\Users\\Public\\_tt\\backtest\\training\\'
outputdir = "{}{}\\".format(outputdir_,strtoday)
if not os.path.exists(outputdir):
    print "create "+outputdir
    os.makedirs(outputdir)
inputdir="c:\\Users\\Public\\_tt\\_data\\henry\\"
print "Input Folder:{},Ouput Folder:{}".format(inputdir,outputdir)

"""
fs=os.listdir(fdir)
d = timedelta(seconds=1)
"""

"""
2013-12-11 15:09:28,BEST_ASK,30.25,56,
2013-12-11 15:09:28,BEST_ASK,30.25,48,
2013-12-11 15:09:28,TRADE,30.16,1465546,OP
"""
def GMT2NYTime(arr):
    tm=''
    trade='0'
    bid='0'
    ask='0'
    askf=0
    bidf=1111
    if arr:
        for line in arr:
            #(t,ty,vl,sz) = spl(i)
            [t,ty,vl,sz,cc]=line.split(',')
            if not tm:
                tm = utc2ny(t)
            if ty=='BEST_ASK':
                tmp=float(vl)
                if tmp > askf:
                    askf=tmp
                    ask = vl
            elif ty=='BEST_BID':
                tmp=float(vl)
                if tmp < bidf:
                    bidf=tmp
                    bid = vl
            #elif ty=='TRADE':
            #    trade = vl
    return (tm,bid,ask)

lb='0'
la='0'

"""
File format: TIME,TYPE,VALUE,SIZE,CC
"""
def GetBidAskFromFile(fpath1, tlower=0):
    print "Fetch file {} to get bid ask info.".format(fpath1)
    askbid=[]
    global la, lb
    with open(fpath1) as f:
        s1=f.readlines()
    s = []
    t = ''
    tstamp = 0
    checktime = True
    for i in s1:
        lss = i[:-1].split(',')
        tm  = lss[0]
        if checktime:
            tstamp = time.mktime(datetime.datetime.strptime(tm, "%Y-%m-%d %H:%M:%S").timetuple())
            if tstamp < tlower:
                continue
            else:
                checktime = False
        if t=='' or t==tm:
            s.append(i[:-1])
        else:
            (t2,b2,a2)=GMT2NYTime(s)
            if b2=='0':b2=lb
            if a2=='0':a2=la
            if a2!='0' and b2!='0' and float(a2)-float(b2)<1:
                askbid.append([t2,b2,a2])
                #print '{},{},{}'.format(t2,b2,a2)
            lb=b2
            la=a2
            s=[i[:-1]]
        t=tm
    return askbid


def __draw(data):
    #return
    BINUM=200
    fig = plt.figure(figsize=(24.0, 20.0))
    c=data.spread
    ax1 = plt.subplot(221)
    symbol=data['master'][0]+'-'+data['slave'][0]
    pictime=str(data.index[0]) + '~' + str(data.index[-1])
    pictime=pictime.replace(':','-')
    pictime=pictime.replace(' ','-')
    plt.title("spread of "+symbol + '@' + pictime)

    plt.setp(ax1.get_xticklabels(), visible=True)
    c.hist(bins=np.arange(c.min(),c.max(),(c.max()-c.min())/BINUM),color='g',alpha=0.4)

    ax2 = plt.subplot(222)
    plt.setp(ax1.get_xticklabels(), visible=True)
    data.spread.plot(ax=ax2,style='--')


    ax3 = plt.subplot(223)
    plt.title("ask(red),bid(blue)")
    plt.setp(ax3.get_xticklabels(), visible=True)
    c=data.ask
    c.hist(bins=np.arange(c.min(),c.max(),(c.max()-c.min())/BINUM),color='r',alpha=0.4)
    plt.setp(ax3.get_xticklabels(), visible=True)
    c=data.bid
    c.hist(bins=np.arange(c.min(),c.max(),(c.max()-c.min())/BINUM),color='b',alpha=0.4)

    ax4 = plt.subplot(224)
    plt.setp(ax4.get_xticklabels(), visible=True)
    data.bid.plot(ax=ax4,style='-')
    data.ask.plot(ax=ax4,style='--')
    ax4.plot(data.ix[data.buy].index, data.ask[data.buy],'^', markersize=15, color='r')
    ax4.plot(data.ix[data.sell].index, data.bid[data.sell],'v', markersize=15, color='r')
    ax4.plot(data.ix[data.liquidate].index, data.bid[data.liquidate],'8', markersize=15, color='r')
    #plt.subplots_adjust(left=0.1, bottom=0.1, wspace=0.09,hspace=0.09)

    #outputdir="c:\\Users\\Public\\_tt\\_data\\henry\\"

    pngfile='{s0}/{fname}.png'.format(s0=outputdir,fname=symbol+'_'+pictime)
    plt.savefig(pngfile)

    #http://stackoverflow.com/questions/21884271/warning-about-too-many-open-figures
    plt.clf()
    plt.close(fig)

    #plt.show()

"""
qstr="('{prev}'<Datetime) & (Datetime<'{curr}')".format(prev=str_previousday,curr=str_currentday)
df = dff.query(qstr)
"""
def draw(fpath, times=[]):
    print fpath
    print times

    header_row=['dt','master','mbid','mask','slave','sbid', 'sask']
    ptotal=pd.read_csv(fpath,index_col=0,parse_dates=True,dayfirst=True, names=header_row)
    ptotal['ask']   = ptotal['mask'] - ptotal['sbid']
    ptotal['bid']   = ptotal['mbid'] - ptotal['sask']
    ptotal['spread']= ptotal['ask'] - ptotal['bid']

    if not times:
        print "get day!"
        return

    for day in times:
        qstr="('{prev}'<=dt) & (dt<='{curr}')".format(prev=day + ' 09:29:59',
                                                      curr=day + ' 16:00:59')
        print qstr
        p1 = ptotal.query(qstr)
        if len(p1)==0:
          continue

        __draw(p1)

def backtest(fpath):
    print fpath

    #outputdir="c:\\Users\\Public\\_tt\\_data\\henry\\"
    f=open(outputdir+"trade.log",'a+')

    header_row=['dt','master','mbid','mask','slave','sbid', 'sask']
    ptotal=pd.read_csv(fpath,index_col=0,parse_dates=True,dayfirst=True, names=header_row)
    ptotal['ask']   = ptotal['mask'] - ptotal['sbid']
    ptotal['bid']   = ptotal['mbid'] - ptotal['sask']
    ptotal['spread']= ptotal['ask'] - ptotal['bid']
    ptotal['buy'] =False
    ptotal['sell']=False
    ptotal['liquidate']=False
    ptotal['pnl']=0



    timeintervals = [10, 60, 240, 480] # 1min, 10min, 30mins, 60mins

    for j in timeintervals:
        ptotal['mvgstd_ab_'+str(j)] = (pd.rolling_std(ptotal.bid,j)+pd.rolling_std(ptotal.ask,j))/2
        ptotal['mvgstd_bid_'+str(j)]= pd.rolling_std(ptotal.bid,j)
        ptotal['mvgstd_ask_'+str(j)]= pd.rolling_std(ptotal.ask,j)
        ptotal['rspread_'+str(j)]   = pd.rolling_max(ptotal.bid,j)-pd.rolling_min(ptotal.ask,j)

    for group in ptotal.groupby(ptotal.index.date):
        d=group[1]
        pictime=str(d.index[0]) + '~' + str(d.index[-1])
        pictime=pictime.replace(':','-')
        #pictime=pictime.replace(' ','-')
        pngfile='{s0}/{fname}.png'.format(s0=outputdir,fname='mvgstd_ab_'+pictime)
        if os.path.exists(pngfile):
            continue
        fname='mvgstd_ab_'+pictime
        d[['mvgstd_ab_10','mvgstd_ab_60','mvgstd_ab_240','mvgstd_ab_480']].plot(title=pictime,figsize=(24.0, 20.0))
        #plt.show()
        plt.savefig(pngfile)
        plt.clf()
        plt.close()

    # Model parameters
    ptotal['STEPSIZE']    = 0.5 + (ptotal['mvgstd_ab_10']+ptotal['mvgstd_ab_10'])/2
    ptotal['RANGEGUESS']  = 1.0
    ptotal['EXPECTEDMEAT']= 1.0


    (NOPOSITION,OPENTRADE_LONG,OPENTRADE_SHORT) = -1,0,1
    (tradeprice_last,tradeprice_avg) = -1,-1
    state = NOPOSITION

    tradeprices=[]
    pnls=[]
    dates=[]

    depth = 0
    cost_commission=0.05
    currentDate=None
    startingIndex=0
    dayDataIndex=0

    (STEPSIZE,RANGEGUESS,EXPECTEDMEAT) = (0.5,1,1)

    """
    >>> type(data)
    Out[1]: pandas.core.frame.DataFrame
    >>> data.index[-1]
    Out[2]: Timestamp('2013-12-11 10:09:28', tz=None)
    >>> data.index[-1].date
    Out[3]: <function date>
    >>> data.index[-1].date()
    Out[4]: datetime.date(2013, 12, 11)
    """

    for i in range(1,len(ptotal)+1):
      data = ptotal[startingIndex:i]

      if currentDate != data.index[-1].date():
          currentDate = data.index[-1].date()
          dates.append(currentDate)
          print "*"*25,str(currentDate),"*"*25
          depth = 0
          dayDataIndex = i-1

      daydata = ptotal[dayDataIndex:i]

      lowerbound = daydata.ask.min()
      upperbound = daydata.bid.max()

      # OPEN POS
      if (upperbound - lowerbound > RANGEGUESS) and (data.ask[-1]==lowerbound or data.bid[-1]==upperbound) and state == NOPOSITION:
        if (data.index[-1].time() < datetime.time(11, 00, 00)) or ((upperbound - lowerbound > (RANGEGUESS+depth*0.5))):
          #print data.bid[-1],data.ask[-1],lowerbound,upperbound
          if data.ask[-1] == lowerbound:
              f.write("LONG@{}@{}\n".format(lowerbound,str(data.index[-1])))
              state = OPENTRADE_LONG
              data.buy[-1]=True
              ptotal.buy[i-1]=True
              tradeprice_avg = lowerbound
              tradeprice_last = lowerbound
              tradeprices.append(tradeprice_last)
          elif data.bid[-1] == upperbound:
              f.write("SHORT@{}@{}\n".format(upperbound,str(data.index[-1])))
              state = OPENTRADE_SHORT
              data.sell[-1]=True
              ptotal.sell[i-1]=True
              tradeprice_last = upperbound
              tradeprice_avg = upperbound
              tradeprices.append(tradeprice_last)
      elif state==OPENTRADE_LONG:
        #if data.ask[-1] < tradeprice_last - STEPSIZE:#0.5 -> to be changed lognormal??
        if data.ask[-1] < tradeprice_last - data.STEPSIZE[-1]:#0.5 -> to be changed lognormal??
            f.write("LONG@{}@{}\n".format(lowerbound,str(data.index[-1])))
            tradeprice_avg = (lowerbound+tradeprice_avg*len(tradeprices))/(len(tradeprices)+1)
            tradeprice_last = lowerbound
            tradeprices.append(tradeprice_last)
            data.buy[-1]=True
            ptotal.buy[i-1]=True
        elif data.bid[-1] > (tradeprice_avg+data.EXPECTEDMEAT[-1]+data.mvgstd_ab_480[-1]) or \
                (upperbound - lowerbound > RANGEGUESS+1.5 and (data.ask[-1]==lowerbound or data.bid[-1]==upperbound) and data.bid[-1] > (tradeprice_avg+cost_commission)):
            f.write("CLOSE_LONG@{}@{}\n".format(data.bid[-1],str(data.index[-1])))
            pnl=len(tradeprices)*data.bid[-1] - sum(tradeprices)
            f.write("PNL:{}\n".format(pnl))

            state = NOPOSITION
            tradeprices=[]
            depth+=1
            pnls.append(pnl)
            (tradeprice_last,tradeprice_avg) = -1,-1
            startingIndex = i
            data.liquidate[-1]=True
            ptotal.liquidate[i-1]=True
            __draw(data)
      elif state==OPENTRADE_SHORT:
        if data.bid[-1] > tradeprice_last + data.STEPSIZE[-1]:
            f.write("SHORT@{}@{}\n".format(upperbound,str(data.index[-1])))
            tradeprice_avg = (upperbound+tradeprice_avg*len(tradeprices))/(len(tradeprices)+1)
            tradeprice_last = upperbound
            tradeprices.append(tradeprice_last)
            data.sell[-1]=True
            ptotal.sell[i-1]=True
        elif data.ask[-1] < (tradeprice_avg-data.EXPECTEDMEAT[-1]-data.mvgstd_ab_480[-1]) or \
                (upperbound - lowerbound > RANGEGUESS+1.5 and (data.ask[-1]==lowerbound or data.bid[-1]==upperbound) and data.ask[-1] < (tradeprice_avg-cost_commission)):
            f.write("CLOSE_SHORT@{}@{}\n".format(data.ask[-1],str(data.index[-1])))
            pnl=sum(tradeprices) - len(tradeprices)*data.ask[-1]
            f.write("PNL:{}\n".format(sum(tradeprices) - len(tradeprices)*data.ask[-1]))

            state = NOPOSITION
            tradeprices=[]
            depth+=1
            pnls.append(pnl)
            (tradeprice_last,tradeprice_avg) = -1,-1
            startingIndex = i
            data.liquidate[-1]=True
            ptotal.liquidate[i-1]=True
            __draw(data)
    #__draw(ptotal)
    #print ptotal.describe()
    f.write("TOTAL PNL:{}\n".format(sum(pnls)))
    #ptotal['rangesize']=

def GenerateSpreadFile():
  masterBO=[]
  slaveBO=[]
  
  ss=[('ATHM','BITA')]#,('HMIN','HTHT')]
  suffix='.csv'
  #outputdir="c:\\Users\\Public\\_tt\\_data\\henry\\"
  for master,slave in ss:
      fpath1=inputdir+master+suffix
      fpath2=inputdir+slave+suffix
      ofilename="{}{}-{}{}".format(outputdir,master,slave,suffix)
      if os.path.exists(ofilename) and os.stat(ofilename)[6]>0:
        backtest(ofilename)
        continue
  
      masterBO = GetBidAskFromFile(fpath1)
      slaveBO = GetBidAskFromFile(fpath2, time.mktime(datetime.datetime.strptime(masterBO[0][0], "%Y-%m-%d %H:%M:%S").timetuple()))
  
      mlen = len(masterBO)
      slen = len(slaveBO)
      index1 = 0
      index2 = 0
  
      hash0=None
      item = [master,0,0,slave,0,0] # bid1, ask1, bid2, ask2
      #2014-03-24 21:34:06,HMIN,HTHT,7.280000,6.550000,11.070958,9.187382,8.352618,6.469042,0.730000
      ofile = open(ofilename,'w')
      days=set()
      while index1 < mlen and index2 < slen:
          try:
              time1 = masterBO[index1][0]
              time2 = slaveBO[index2][0]

              tmp1=time1.split(' ')
              tmp2=time2.split(' ')

              # remove data before 9:30 and after 16:00

              h_=int(tmp1[1][0:2])
              m_=int(tmp1[1][3:5])
              if (m_<30 if h_==9 else (h_<9)) or h_>=16:
                  index1 += 1
                  continue
              h_=int(tmp2[1][0:2])
              m_=int(tmp2[1][3:5])
              if (m_<30 if h_==9 else (h_<9)) or h_>=16:
                  index2 += 1
                  continue
  
              days.add(tmp1[0])
              #days.add(tmp2[0])
  
              t1 = time.mktime(datetime.datetime.strptime(time1, "%Y-%m-%d %H:%M:%S").timetuple())
              t2 = time.mktime(datetime.datetime.strptime(time2, "%Y-%m-%d %H:%M:%S").timetuple())

              if t1 < t2:
                  timestamp = time1
                  item[1] = masterBO[index1][1]
                  item[2] = masterBO[index1][2]
                  index1 += 1
              elif t1 > t2:
                  timestamp = time2
                  item[4] = slaveBO[index2][1]
                  item[5] = slaveBO[index2][2]
                  index2 += 1
              else:
                  timestamp = time1
                  item[1] = masterBO[index1][1]
                  item[2] = masterBO[index1][2]
                  item[4] = slaveBO[index2][1]
                  item[5] = slaveBO[index2][2]
                  index2 += 1
                  index1 += 1
  
  
              #if (float(item[2])-float(item[1])>1 or float(item[5])-float(item[4])>1):
              #    continue
    
              #if not item[1] and not item[2] and not item[4] and not item[5] and item0!=item:
              if item[1] and item[2] and item[4] and item[5] and hash0!=hash(str(item)):
                  s=",".join(item)
                  ofile.write("{},{}\n".format(timestamp, s))
                  hash0 = hash(str(item))
          except Exception as e:
              s = str(e)
              print "Exception:",s
              print "Master:", index1, masterBO[index1]
              print "Slave:", index2, slaveBO[index2]
      ofile.close()
  
      # split file
  
  
      #draw(ofilename, days)

if __name__ == '__main__':
    GenerateSpreadFile()
